# -*- coding = utf-8 -*-
# @Time : 2021/10/12 8:41
# @Author :Henry L
# @File : Jieba.py
# @Software : PyCharm
# @Mean : 将海量数据进行批量解析处理
import csv
import pickle
from Data_compare import StringCompareUtil
import jieba
import time
from Insert_to_dictionary import InsertData
import os
# jieba.load_userdict("../Data/newdict.txt")

Administrative_divisions = [
    '省', '市', '自治区', '地区', '自治州', '盟', '自治县', '旗', '自治旗', '特区', '林区', '街道',
    '镇', '民族乡', '乡', '民族苏木', '苏木', '县', '村', '屯', '社区', '区', '街办', '科技','管理','有限责任','保健站','分院','疾病',
    '卫生室', '中西医', '蒙医', '中医', '西医', '诊所', '卫生院', '医务室', '藏医', '服务中心', '口腔', '医疗', '门诊','医馆','分店',
    '美容', '综合', '有限公司', '监督所', '牙科', '内科', '（', '）', '卫生所', '医院', '卫生局', '卫生','保安','新城','职业','学校',
    '皮肤科', '妇科', '服务站', '妇幼保健', '血液', '中心', '透析', '康复中心', '郭家', '城东', '办事处', '龙泉', '高家','城关','合伙',
    '公司','护理','结合','经济','技术开发','防治所','肿瘤','第二','家庄','医师','协会','药房','连锁','分公司','坐堂','咨询','专科','预防',
    '标准化','委会','新安','医学','投资','康复','防治','计划生育','技术','指导','结合部','眼科','开发','高新','家务','李家','东方','第三',
    '控制中心','平安','第四','第一','第五','普通','龙山','赵家','小河','各庄','和平','大王','医药','莲花','城西','农场','儿科','学院',
    '高新技术','个体','健康','计生','服务室','外科','职工','集团','人民','预防','分室','幸福','仁爱','甲级','控制','医生','中学','花园',
    '服务','太平','行政','兴隆','诊室','新街','康宁','机关','现代','按摩','旅游','店子','河东','黄家','人口','药业','股份','新华','国际',
    '阳光','妇幼','保健','附属','一体化','有限','工业','分所','监督','集体','幼儿园','诊疗','瑶族','山东','山西','小学','同仁堂','河南',
    '委员会','民族','堂部','家园', '友谊','整形','妇产','惠民','骨伤科','药店','中国','上海浦东','皮肤病','北京','大学','码头','发展','联合',
    '仁和', '第六', '理疗', '新疆生产建设兵团','精神病', '站', '部', '控股','国药','平民','漱玉大'
]
s = StringCompareUtil()
start_time = time.time()
D = InsertData()


with open("../Data/比对是否重复1205.csv", newline='', encoding='utf-8') as file:
    data = csv.reader(file)
    headers_row = next(data)
    num = 1
    # 从第二行开始进行分词
    for row in data:
        print(num)
        # 先只分名称
        name = row[0]
        for i in Administrative_divisions:
            name = name.replace(i, '')
        provinceidname = row[1]
        cityidname = row[2]
        countyidname = row[3]
        seg_list = jieba.cut(name, cut_all=False)
        # print(seg_list)
        # 去掉前缀和后缀
        for i in seg_list:
            # 将省份、城市、乡村的名字剔除在HashTable外面
            if i == provinceidname or i in provinceidname:
                # 不做任何处理==删除
                continue
            elif i == cityidname or i in cityidname:
                continue
            elif i == countyidname or i in countyidname:
                continue
            elif len(i) <= 1:
                continue
            # if len(i) <= 1:
            #     continue
            else:
                # 加进hash table
                D.insert(i, row)
        num = num + 1
file.close()
with open('dict.pkl', 'wb') as tf:
    pickle.dump(D.data, tf)
end_time = time.time()

# print(D.data)
for i in D.data.keys():
    data_list = D.data[i]
    # if len(data_list) > 600:
        # print(i)
sting = StringCompareUtil()
# print(end_time-start_time)
header_row = ['name1', 'name2']
with open("../Data/Repeated_data.csv", 'a+', encoding='utf-8', newline='') as file1:
    num = 1
    csv_writer = csv.writer(file1)
    csv_writer.writerow(header_row)
    for i in D.data.keys():
        num = num + 1
        print(i, len(D.data[i]), num)
        data_list = D.data[i]
        # if num < 2716:
        #     continue
        if len(data_list) > 500:
            continue
        if len(data_list) == 1:
            continue
        for num1 in range(0, len(D.data[i])):
            data1 = data_list[num1]
            # if data1[5] != '1':
            #     continue
            for num2 in range(num1+1, len(D.data[i])):
                data2 = data_list[num2]
                    # if data2[5] == '0':
                        # 比较名称和省份
                if data1[1] != data2[1] or data1[0] == data2[0]:
                    continue
                if data1[5] == '1' and data2[5] == '0':
                # if 1 == 1:
                    name_result = sting.compare(data1[0], data2[0])
                    address_result = 0
                    if data1[4] != '' and data2[4] != '':
                        address_result = sting.compare(data1[4], data2[4])
                    if (name_result == 1 and '房' not in data1[0] and '店' not in data1[0] and
                            '房' not in data2[0] and '店' not in data2[0]) or address_result == 1:
                        csv_writer.writerow(data1[0:5] + data2[0:5] + [name_result, address_result])
                    else:
                        continue
                    # if name_result == 1 or address_result == 1:
                    #     csv_writer.writerow(data1 + data2 + [name_result, address_result])
            # else:
            #     continue
file1.close()
end_time = time.time()
print(end_time-start_time)



